{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda2/envs/py36/lib/python3.6/site-packages/lightgbm/__init__.py:47: UserWarning: Starting from version 2.2.1, the library file in distribution wheels for macOS is built by the Apple Clang (Xcode_8.3.3) compiler.\n",
      "This means that in case of installing LightGBM from PyPI via the ``pip install lightgbm`` command, you don't need to install the gcc compiler anymore.\n",
      "Instead of that, you need to install the OpenMP library, which is required for running LightGBM on the system with the Apple Clang compiler.\n",
      "You can install the OpenMP library by the following command: ``brew install libomp``.\n",
      "  \"You can install the OpenMP library by the following command: ``brew install libomp``.\", UserWarning)\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import lightgbm as lgb\n",
    "from sklearn.model_selection import KFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>55</th>\n",
       "      <th>56</th>\n",
       "      <th>57</th>\n",
       "      <th>58</th>\n",
       "      <th>59</th>\n",
       "      <th>60</th>\n",
       "      <th>61</th>\n",
       "      <th>62</th>\n",
       "      <th>63</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4.0</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>-8.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-8.0</td>\n",
       "      <td>-15.0</td>\n",
       "      <td>-8.0</td>\n",
       "      <td>-19.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-9.0</td>\n",
       "      <td>...</td>\n",
       "      <td>4.0</td>\n",
       "      <td>-24.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>-6.0</td>\n",
       "      <td>-20.0</td>\n",
       "      <td>-26.0</td>\n",
       "      <td>-44.0</td>\n",
       "      <td>-27.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>12.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>89.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>-18.0</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-14.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-7.0</td>\n",
       "      <td>-8.0</td>\n",
       "      <td>-18.0</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>-6.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>18.0</td>\n",
       "      <td>-8.0</td>\n",
       "      <td>-6.0</td>\n",
       "      <td>-5.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>-6.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>...</td>\n",
       "      <td>8.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>-10.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>51.0</td>\n",
       "      <td>92.0</td>\n",
       "      <td>45.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>-4.0</td>\n",
       "      <td>-6.0</td>\n",
       "      <td>-8.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-5.0</td>\n",
       "      <td>-8.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>-4.0</td>\n",
       "      <td>-68.0</td>\n",
       "      <td>-5.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>-11.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>-8.0</td>\n",
       "      <td>-21.0</td>\n",
       "      <td>...</td>\n",
       "      <td>3.0</td>\n",
       "      <td>-7.0</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-5.0</td>\n",
       "      <td>-6.0</td>\n",
       "      <td>-15.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 65 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      0     1    2    3     4     5     6     7     8     9  ...    55    56  \\\n",
       "0   4.0  -2.0 -8.0 -1.0  -8.0 -15.0  -8.0 -19.0   0.0  -9.0  ...   4.0 -24.0   \n",
       "1  12.0   5.0  5.0  2.0  12.0   1.0  89.0  17.0 -18.0  -3.0  ... -14.0   1.0   \n",
       "2  18.0  -8.0 -6.0 -5.0   7.0  16.0  10.0  25.0  -6.0   4.0  ...   8.0  34.0   \n",
       "3  -1.0  -4.0 -6.0 -8.0  17.0   2.0  14.0   6.0   4.0  -1.0  ...  -5.0  -8.0   \n",
       "4   3.0  11.0  1.0 -5.0   4.0 -11.0  -1.0   2.0  -8.0 -21.0  ...   3.0  -7.0   \n",
       "\n",
       "     57   58    59    60    61    62    63  label  \n",
       "0  12.0 -2.0  -6.0 -20.0 -26.0 -44.0 -27.0      0  \n",
       "1  -7.0 -8.0 -18.0  -3.0   4.0  62.0  -6.0      0  \n",
       "2   8.0  5.0 -10.0  17.0  51.0  92.0  45.0      0  \n",
       "3   0.0 -2.0   0.0   6.0  -4.0 -68.0  -5.0      0  \n",
       "4  -3.0 -1.0  -5.0  -6.0 -15.0  11.0   0.0      0  \n",
       "\n",
       "[5 rows x 65 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "columns = list(range(64))\n",
    "names = list(range(64))\n",
    "names.append('label')\n",
    "train = pd.read_csv('../data/train/0.csv', header=0, names=names)\n",
    "tmp = pd.read_csv('../data/train/1.csv', header=0, names=names)\n",
    "train = pd.concat([train, tmp], axis=0)\n",
    "tmp = pd.read_csv('../data/train/2.csv', header=0, names=names)\n",
    "train = pd.concat([train, tmp], axis=0)\n",
    "tmp = pd.read_csv('../data/train/3.csv', header=0, names=names)\n",
    "train = pd.concat([train, tmp], axis=0)\n",
    "test = pd.read_csv('../data/test/test.csv', names=columns)\n",
    "train = train.reset_index()\n",
    "train.drop('index', axis=1, inplace=True)\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fold 1\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's multi_logloss: 0.219158\tvalid_1's multi_logloss: 0.33058\n",
      "[200]\ttraining's multi_logloss: 0.0668168\tvalid_1's multi_logloss: 0.191028\n",
      "[300]\ttraining's multi_logloss: 0.0323947\tvalid_1's multi_logloss: 0.152031\n",
      "[400]\ttraining's multi_logloss: 0.0208681\tvalid_1's multi_logloss: 0.137182\n",
      "[500]\ttraining's multi_logloss: 0.0156428\tvalid_1's multi_logloss: 0.13043\n",
      "[600]\ttraining's multi_logloss: 0.0127188\tvalid_1's multi_logloss: 0.127166\n",
      "[700]\ttraining's multi_logloss: 0.0108715\tvalid_1's multi_logloss: 0.125078\n",
      "[800]\ttraining's multi_logloss: 0.00960584\tvalid_1's multi_logloss: 0.123262\n",
      "[900]\ttraining's multi_logloss: 0.00867146\tvalid_1's multi_logloss: 0.12244\n",
      "[1000]\ttraining's multi_logloss: 0.00796345\tvalid_1's multi_logloss: 0.121188\n",
      "[1100]\ttraining's multi_logloss: 0.00740826\tvalid_1's multi_logloss: 0.120432\n",
      "[1200]\ttraining's multi_logloss: 0.00696395\tvalid_1's multi_logloss: 0.119806\n",
      "[1300]\ttraining's multi_logloss: 0.00660441\tvalid_1's multi_logloss: 0.119363\n",
      "[1400]\ttraining's multi_logloss: 0.00630664\tvalid_1's multi_logloss: 0.119028\n",
      "[1500]\ttraining's multi_logloss: 0.00605618\tvalid_1's multi_logloss: 0.118921\n",
      "[1600]\ttraining's multi_logloss: 0.00584326\tvalid_1's multi_logloss: 0.118864\n",
      "[1700]\ttraining's multi_logloss: 0.00565135\tvalid_1's multi_logloss: 0.118798\n",
      "[1800]\ttraining's multi_logloss: 0.00548534\tvalid_1's multi_logloss: 0.118605\n",
      "[1900]\ttraining's multi_logloss: 0.00533322\tvalid_1's multi_logloss: 0.118489\n",
      "[2000]\ttraining's multi_logloss: 0.00519982\tvalid_1's multi_logloss: 0.118453\n",
      "[2100]\ttraining's multi_logloss: 0.00507767\tvalid_1's multi_logloss: 0.118385\n",
      "[2200]\ttraining's multi_logloss: 0.0049661\tvalid_1's multi_logloss: 0.118372\n",
      "[2300]\ttraining's multi_logloss: 0.00486451\tvalid_1's multi_logloss: 0.118213\n",
      "Early stopping, best iteration is:\n",
      "[2257]\ttraining's multi_logloss: 0.00490731\tvalid_1's multi_logloss: 0.118113\n",
      "fold 2\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's multi_logloss: 0.22015\tvalid_1's multi_logloss: 0.326702\n",
      "[200]\ttraining's multi_logloss: 0.0674472\tvalid_1's multi_logloss: 0.18879\n",
      "[300]\ttraining's multi_logloss: 0.0326094\tvalid_1's multi_logloss: 0.151871\n",
      "[400]\ttraining's multi_logloss: 0.0210071\tvalid_1's multi_logloss: 0.138178\n",
      "[500]\ttraining's multi_logloss: 0.0157122\tvalid_1's multi_logloss: 0.132035\n",
      "[600]\ttraining's multi_logloss: 0.0127303\tvalid_1's multi_logloss: 0.129381\n",
      "[700]\ttraining's multi_logloss: 0.0108704\tvalid_1's multi_logloss: 0.127557\n",
      "[800]\ttraining's multi_logloss: 0.00960705\tvalid_1's multi_logloss: 0.126578\n",
      "[900]\ttraining's multi_logloss: 0.00868081\tvalid_1's multi_logloss: 0.125894\n",
      "[1000]\ttraining's multi_logloss: 0.00797034\tvalid_1's multi_logloss: 0.125495\n",
      "[1100]\ttraining's multi_logloss: 0.00741029\tvalid_1's multi_logloss: 0.124927\n",
      "[1200]\ttraining's multi_logloss: 0.00697434\tvalid_1's multi_logloss: 0.124529\n",
      "[1300]\ttraining's multi_logloss: 0.00661877\tvalid_1's multi_logloss: 0.124344\n",
      "[1400]\ttraining's multi_logloss: 0.00631924\tvalid_1's multi_logloss: 0.124171\n",
      "[1500]\ttraining's multi_logloss: 0.00606254\tvalid_1's multi_logloss: 0.124125\n",
      "[1600]\ttraining's multi_logloss: 0.00584806\tvalid_1's multi_logloss: 0.12385\n",
      "[1700]\ttraining's multi_logloss: 0.00565767\tvalid_1's multi_logloss: 0.123755\n",
      "Early stopping, best iteration is:\n",
      "[1683]\ttraining's multi_logloss: 0.00568942\tvalid_1's multi_logloss: 0.123693\n",
      "fold 3\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's multi_logloss: 0.22133\tvalid_1's multi_logloss: 0.327153\n",
      "[200]\ttraining's multi_logloss: 0.0678737\tvalid_1's multi_logloss: 0.187121\n",
      "[300]\ttraining's multi_logloss: 0.0329789\tvalid_1's multi_logloss: 0.149368\n",
      "[400]\ttraining's multi_logloss: 0.0212339\tvalid_1's multi_logloss: 0.136272\n",
      "[500]\ttraining's multi_logloss: 0.0158494\tvalid_1's multi_logloss: 0.130275\n",
      "[600]\ttraining's multi_logloss: 0.0128807\tvalid_1's multi_logloss: 0.126997\n",
      "[700]\ttraining's multi_logloss: 0.0110049\tvalid_1's multi_logloss: 0.125173\n",
      "[800]\ttraining's multi_logloss: 0.00971719\tvalid_1's multi_logloss: 0.123566\n",
      "[900]\ttraining's multi_logloss: 0.00879541\tvalid_1's multi_logloss: 0.12276\n",
      "[1000]\ttraining's multi_logloss: 0.00809434\tvalid_1's multi_logloss: 0.121899\n",
      "[1100]\ttraining's multi_logloss: 0.00752829\tvalid_1's multi_logloss: 0.12163\n",
      "[1200]\ttraining's multi_logloss: 0.00707083\tvalid_1's multi_logloss: 0.121088\n",
      "[1300]\ttraining's multi_logloss: 0.00670748\tvalid_1's multi_logloss: 0.120797\n",
      "[1400]\ttraining's multi_logloss: 0.00639868\tvalid_1's multi_logloss: 0.120506\n",
      "[1500]\ttraining's multi_logloss: 0.00614159\tvalid_1's multi_logloss: 0.120561\n",
      "Early stopping, best iteration is:\n",
      "[1448]\ttraining's multi_logloss: 0.00626822\tvalid_1's multi_logloss: 0.12033\n",
      "fold 4\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's multi_logloss: 0.22256\tvalid_1's multi_logloss: 0.316863\n",
      "[200]\ttraining's multi_logloss: 0.0687816\tvalid_1's multi_logloss: 0.1792\n",
      "[300]\ttraining's multi_logloss: 0.0331878\tvalid_1's multi_logloss: 0.13995\n",
      "[400]\ttraining's multi_logloss: 0.0213811\tvalid_1's multi_logloss: 0.124645\n",
      "[500]\ttraining's multi_logloss: 0.0160052\tvalid_1's multi_logloss: 0.117404\n",
      "[600]\ttraining's multi_logloss: 0.0130163\tvalid_1's multi_logloss: 0.112975\n",
      "[700]\ttraining's multi_logloss: 0.0111231\tvalid_1's multi_logloss: 0.110094\n",
      "[800]\ttraining's multi_logloss: 0.00981571\tvalid_1's multi_logloss: 0.108477\n",
      "[900]\ttraining's multi_logloss: 0.00888167\tvalid_1's multi_logloss: 0.107051\n",
      "[1000]\ttraining's multi_logloss: 0.00815718\tvalid_1's multi_logloss: 0.106298\n",
      "[1100]\ttraining's multi_logloss: 0.00757333\tvalid_1's multi_logloss: 0.105794\n",
      "[1200]\ttraining's multi_logloss: 0.00711343\tvalid_1's multi_logloss: 0.105141\n",
      "[1300]\ttraining's multi_logloss: 0.00673045\tvalid_1's multi_logloss: 0.104701\n",
      "[1400]\ttraining's multi_logloss: 0.00642096\tvalid_1's multi_logloss: 0.104354\n",
      "[1500]\ttraining's multi_logloss: 0.00616135\tvalid_1's multi_logloss: 0.104102\n",
      "[1600]\ttraining's multi_logloss: 0.00593747\tvalid_1's multi_logloss: 0.103932\n",
      "[1700]\ttraining's multi_logloss: 0.00574396\tvalid_1's multi_logloss: 0.103783\n",
      "[1800]\ttraining's multi_logloss: 0.00557333\tvalid_1's multi_logloss: 0.103425\n",
      "[1900]\ttraining's multi_logloss: 0.00542095\tvalid_1's multi_logloss: 0.103425\n",
      "[2000]\ttraining's multi_logloss: 0.00528596\tvalid_1's multi_logloss: 0.103291\n",
      "[2100]\ttraining's multi_logloss: 0.00515907\tvalid_1's multi_logloss: 0.103255\n",
      "Early stopping, best iteration is:\n",
      "[2067]\ttraining's multi_logloss: 0.00520013\tvalid_1's multi_logloss: 0.103182\n",
      "fold 5\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's multi_logloss: 0.221123\tvalid_1's multi_logloss: 0.329053\n",
      "[200]\ttraining's multi_logloss: 0.0677739\tvalid_1's multi_logloss: 0.188151\n",
      "[300]\ttraining's multi_logloss: 0.0329365\tvalid_1's multi_logloss: 0.150228\n",
      "[400]\ttraining's multi_logloss: 0.0212825\tvalid_1's multi_logloss: 0.135814\n",
      "[500]\ttraining's multi_logloss: 0.0159168\tvalid_1's multi_logloss: 0.129117\n",
      "[600]\ttraining's multi_logloss: 0.0129263\tvalid_1's multi_logloss: 0.124785\n",
      "[700]\ttraining's multi_logloss: 0.0110467\tvalid_1's multi_logloss: 0.121965\n",
      "[800]\ttraining's multi_logloss: 0.00972704\tvalid_1's multi_logloss: 0.120477\n",
      "[900]\ttraining's multi_logloss: 0.00880343\tvalid_1's multi_logloss: 0.119736\n",
      "[1000]\ttraining's multi_logloss: 0.00807613\tvalid_1's multi_logloss: 0.118675\n",
      "[1100]\ttraining's multi_logloss: 0.00751746\tvalid_1's multi_logloss: 0.118235\n",
      "[1200]\ttraining's multi_logloss: 0.00706772\tvalid_1's multi_logloss: 0.1179\n",
      "[1300]\ttraining's multi_logloss: 0.0067075\tvalid_1's multi_logloss: 0.117525\n",
      "[1400]\ttraining's multi_logloss: 0.00640156\tvalid_1's multi_logloss: 0.117404\n",
      "[1500]\ttraining's multi_logloss: 0.00614515\tvalid_1's multi_logloss: 0.117199\n",
      "[1600]\ttraining's multi_logloss: 0.00592452\tvalid_1's multi_logloss: 0.117075\n",
      "[1700]\ttraining's multi_logloss: 0.00572805\tvalid_1's multi_logloss: 0.116862\n",
      "[1800]\ttraining's multi_logloss: 0.00556132\tvalid_1's multi_logloss: 0.116437\n",
      "[1900]\ttraining's multi_logloss: 0.00541049\tvalid_1's multi_logloss: 0.116268\n",
      "[2000]\ttraining's multi_logloss: 0.00527016\tvalid_1's multi_logloss: 0.116005\n",
      "[2100]\ttraining's multi_logloss: 0.00514368\tvalid_1's multi_logloss: 0.115778\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2200]\ttraining's multi_logloss: 0.00502826\tvalid_1's multi_logloss: 0.11566\n",
      "[2300]\ttraining's multi_logloss: 0.00491936\tvalid_1's multi_logloss: 0.115627\n",
      "[2400]\ttraining's multi_logloss: 0.00482029\tvalid_1's multi_logloss: 0.115534\n",
      "[2500]\ttraining's multi_logloss: 0.00473299\tvalid_1's multi_logloss: 0.115455\n",
      "Early stopping, best iteration is:\n",
      "[2468]\ttraining's multi_logloss: 0.00475944\tvalid_1's multi_logloss: 0.115409\n"
     ]
    }
   ],
   "source": [
    "x_train = train[columns]\n",
    "y_train = train['label']\n",
    "x_test = test[columns]\n",
    "num_labels = 4\n",
    "\n",
    "params = {\n",
    "    'num_leaves': 60,\n",
    "    'min_data_in_leaf': 30,\n",
    "    'objective': 'multiclass',\n",
    "    'num_class': num_labels,\n",
    "    'max_depth': -1,\n",
    "    'learning_rate': 0.03,\n",
    "    \"min_sum_hessian_in_leaf\": 6,\n",
    "    \"boosting\": \"gbdt\",\n",
    "    \"feature_fraction\": 0.9,\n",
    "    \"bagging_freq\": 1,\n",
    "    \"bagging_fraction\": 0.8,\n",
    "    \"bagging_seed\": 11,\n",
    "    \"lambda_l1\": 0.1,\n",
    "    \"verbosity\": -1,\n",
    "    \"nthread\": 15,\n",
    "    'metric': 'multi_logloss',\n",
    "    \"random_state\": 0,}\n",
    "    # 'device': 'gpu' }\n",
    "\n",
    "\n",
    "folds = KFold(n_splits=5, shuffle=True, random_state=0)\n",
    "oof = np.zeros((x_train.shape[0], num_labels))\n",
    "preds_prob = np.zeros((x_test.shape[0], num_labels))\n",
    "\n",
    "## train and predict\n",
    "feature_importance_df = pd.DataFrame()\n",
    "for fold_, (trn_idx, val_idx) in enumerate(folds.split(x_train)):\n",
    "    print(\"fold {}\".format(fold_ + 1))\n",
    "    trn_data = lgb.Dataset(x_train.iloc[trn_idx], label=y_train.iloc[trn_idx])\n",
    "    val_data = lgb.Dataset(x_train.iloc[val_idx], label=y_train.iloc[val_idx])\n",
    "#     trn_data = lgb.Dataset(x_train[trn_idx], label=y_train[trn_idx])\n",
    "#     val_data = lgb.Dataset(x_train[val_idx], label=y_train[val_idx])\n",
    "\n",
    "    clf = lgb.train(\n",
    "        params,\n",
    "        trn_data,\n",
    "        valid_sets=[trn_data, val_data],\n",
    "        num_boost_round = 10000,\n",
    "        verbose_eval = 100,\n",
    "        early_stopping_rounds = 100)\n",
    "    oof[val_idx] = clf.predict(x_train.iloc[val_idx], num_iteration=clf.best_iteration)\n",
    "\n",
    "    preds_prob += clf.predict(x_test, num_iteration=clf.best_iteration) / folds.n_splits\n",
    "    \n",
    "# preds = np.argmax(preds_prob, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>pred</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  pred\n",
       "0   1     3\n",
       "1   2     2\n",
       "2   3     0\n",
       "3   4     1\n",
       "4   5     0"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds = np.argmax(preds_prob, axis=1)\n",
    "submission = pd.DataFrame({'id': range(len(preds)), 'pred': preds})\n",
    "submission['id'] = submission['id'] + 1\n",
    "submission.to_csv(\"../data/ml_submission.csv\", index=False, header=False)\n",
    "submission.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
